{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a3844223",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "99fff94b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-30 21:20:02.496117: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import json\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f64e3be2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['kelantan-phoneme.json',\n",
       " 'johor-phoneme.json',\n",
       " 'melaka-phoneme.json',\n",
       " 'penang-phoneme.json',\n",
       " 'pahang-phoneme.json',\n",
       " 'kedah-phoneme.json',\n",
       " 'negeri-sembilan-phoneme.json']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phonemes = glob('*-phoneme.json')\n",
    "phonemes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1c787af3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "replace_chars = '.,!?['\n",
    "left, right = [], []\n",
    "for f in phonemes:\n",
    "    with open(f) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    for k, v in data.items():\n",
    "        l = k\n",
    "        for c in replace_chars:\n",
    "            l = l.replace(c, f' ')\n",
    "        l = l.lower()\n",
    "        l = re.sub(r'[ ]+', ' ', l).strip()\n",
    "        r = re.sub(r'[ ]+', ' ', v.replace(']', ' ')).strip()\n",
    "        if len(l) and len(r):\n",
    "            left.append(l)\n",
    "            right.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "98c4fe31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PAD = 0\n",
    "EOS = 1\n",
    "UNK = 2\n",
    "GO = 3\n",
    "[PAD, EOS, UNK, GO]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1a0025bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "left_dict = [PAD, EOS, UNK, GO] + sorted(set(list(''.join(left))))\n",
    "left_dict = {c: no for no, c in enumerate(left_dict)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "34453bf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "right_dict = [PAD, EOS, UNK, GO] + sorted(set(list(''.join(right))))\n",
    "right_dict = {c: no for no, c in enumerate(right_dict)}\n",
    "rev_right_dict = {v: k for k, v in right_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "945890c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0,\n",
       " 1: 1,\n",
       " 2: 2,\n",
       " 3: 3,\n",
       " ' ': 4,\n",
       " \"'\": 5,\n",
       " '-': 6,\n",
       " '`': 7,\n",
       " 'a': 8,\n",
       " 'b': 9,\n",
       " 'c': 10,\n",
       " 'd': 11,\n",
       " 'e': 12,\n",
       " 'f': 13,\n",
       " 'g': 14,\n",
       " 'h': 15,\n",
       " 'i': 16,\n",
       " 'j': 17,\n",
       " 'k': 18,\n",
       " 'l': 19,\n",
       " 'm': 20,\n",
       " 'n': 21,\n",
       " 'o': 22,\n",
       " 'p': 23,\n",
       " 'q': 24,\n",
       " 'r': 25,\n",
       " 's': 26,\n",
       " 't': 27,\n",
       " 'u': 28,\n",
       " 'w': 29,\n",
       " 'y': 30,\n",
       " 'z': 31}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "left_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "e4c68cbd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0,\n",
       " 1: 1,\n",
       " 2: 2,\n",
       " 3: 3,\n",
       " ' ': 4,\n",
       " ')': 5,\n",
       " ',': 6,\n",
       " '.': 7,\n",
       " '/': 8,\n",
       " ':': 9,\n",
       " 'A': 10,\n",
       " 'C': 11,\n",
       " 'E': 12,\n",
       " 'I': 13,\n",
       " 'N': 14,\n",
       " 'S': 15,\n",
       " 'Z': 16,\n",
       " 'a': 17,\n",
       " 'b': 18,\n",
       " 'd': 19,\n",
       " 'e': 20,\n",
       " 'f': 21,\n",
       " 'g': 22,\n",
       " 'h': 23,\n",
       " 'i': 24,\n",
       " 'j': 25,\n",
       " 'k': 26,\n",
       " 'l': 27,\n",
       " 'm': 28,\n",
       " 'n': 29,\n",
       " 'o': 30,\n",
       " 'p': 31,\n",
       " 'r': 32,\n",
       " 's': 33,\n",
       " 't': 34,\n",
       " 'u': 35,\n",
       " 'w': 36,\n",
       " 'z': 37,\n",
       " '\\x8d': 38,\n",
       " '«': 39,\n",
       " '\\xad': 40,\n",
       " 'Ä': 41,\n",
       " 'Ò': 42,\n",
       " 'Ö': 43,\n",
       " 'â': 44,\n",
       " 'ø': 45,\n",
       " 'ù': 46}"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "right_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0de00f6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Translator:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size,\n",
    "                 from_dict_size, to_dict_size, learning_rate, beam_width = 10):\n",
    "        \n",
    "        def cells(reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size_layer,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        def attention(encoder_out, seq_len, reuse=False):\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(num_units = size_layer, \n",
    "                                                                    memory = encoder_out,\n",
    "                                                                    memory_sequence_length = seq_len)\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells(reuse) for _ in range(num_layers)]), \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        encoder_embedding = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        decoder_embedding = tf.Variable(tf.random_uniform([to_dict_size, embedded_size], -1, 1))\n",
    "        \n",
    "        encoder_out, encoder_state = tf.nn.dynamic_rnn(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "            inputs = tf.nn.embedding_lookup(encoder_embedding, self.X),\n",
    "            sequence_length = self.X_seq_len,\n",
    "            dtype = tf.float32)\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        dense = tf.layers.Dense(to_dict_size)\n",
    "        \n",
    "        with tf.variable_scope('decode'):\n",
    "            decoder_cells = attention(encoder_out, self.X_seq_len)\n",
    "            training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(decoder_embedding, decoder_input),\n",
    "                sequence_length = self.Y_seq_len,\n",
    "                time_major = False)\n",
    "            training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                    cell = decoder_cells,\n",
    "                    helper = training_helper,\n",
    "                    initial_state = decoder_cells.zero_state(batch_size, tf.float32).clone(cell_state=encoder_state),\n",
    "                    output_layer = dense)\n",
    "            training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = training_decoder,\n",
    "                    impute_finished = True,\n",
    "                    maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "            self.training_logits = training_decoder_output.rnn_output\n",
    "            \n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            predicting_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(\n",
    "                embedding = decoder_embedding,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS)\n",
    "            predicting_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                    cell = decoder_cells,\n",
    "                    helper = predicting_helper,\n",
    "                    initial_state = decoder_cells.zero_state(batch_size, tf.float32).clone(cell_state=encoder_state),\n",
    "                    output_layer = dense)\n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = predicting_decoder,\n",
    "                    impute_finished = True,\n",
    "                    maximum_iterations = 2 * tf.reduce_max(self.X_seq_len))\n",
    "            self.greedy = predicting_decoder_output.sample_id\n",
    "            self.greedy = tf.identity(self.greedy,name='greedy')\n",
    "        \n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            \n",
    "            encoder_out_tiled = tf.contrib.seq2seq.tile_batch(encoder_out, beam_width)\n",
    "            encoder_state_tiled = tf.contrib.seq2seq.tile_batch(encoder_state, beam_width)\n",
    "            X_seq_len_tiled = tf.contrib.seq2seq.tile_batch(self.X_seq_len, beam_width)\n",
    "            decoder_cell = attention(encoder_out_tiled, X_seq_len_tiled, reuse=True)\n",
    "            \n",
    "            predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = decoder_cell,\n",
    "                embedding = decoder_embedding,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = decoder_cell.zero_state(batch_size * beam_width, tf.float32).clone(\n",
    "                    cell_state = encoder_state_tiled),\n",
    "                beam_width = beam_width,\n",
    "                output_layer = dense,\n",
    "                length_penalty_weight = 0.0)\n",
    "            \n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = tf.reduce_max(self.X_seq_len))\n",
    "            \n",
    "            self.beam = predicting_decoder_output.predicted_ids[:, :, 0]\n",
    "            self.beam = tf.identity(self.beam,name='beam')\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.masks = masks\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7197fd33",
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 256\n",
    "num_layers = 2\n",
    "embedded_size = 256\n",
    "learning_rate = 1e-3\n",
    "batch_size = 32\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3a034214",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_212260/221865152.py:1: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_212260/221865152.py:2: The name tf.InteractiveSession is deprecated. Please use tf.compat.v1.InteractiveSession instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_212260/2944119294.py:17: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_212260/2944119294.py:23: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_212260/2944119294.py:33: The name tf.layers.Dense is deprecated. Please use tf.compat.v1.layers.Dense instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_212260/2944119294.py:35: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-30 21:20:03.026855: I tensorflow/core/platform/profile_utils/cpu_utils.cc:109] CPU Frequency: 2496000000 Hz\n",
      "2022-05-30 21:20:03.027345: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3cd3d40 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2022-05-30 21:20:03.027358: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2022-05-30 21:20:03.028390: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n",
      "2022-05-30 21:20:03.030529: E tensorflow/stream_executor/cuda/cuda_driver.cc:282] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
      "2022-05-30 21:20:03.030545: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:169] retrieving CUDA diagnostic information for host: huseincomel-desktop\n",
      "2022-05-30 21:20:03.030549: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:176] hostname: huseincomel-desktop\n",
      "2022-05-30 21:20:03.030577: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:200] libcuda reported version is: 470.129.6\n",
      "2022-05-30 21:20:03.030591: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:204] kernel reported version is: 470.129.6\n",
      "2022-05-30 21:20:03.030594: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:310] kernel version seems to match DSO: 470.129.6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_212260/2944119294.py:100: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_212260/221865152.py:4: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Translator(size_layer, num_layers, embedded_size, len(left_dict), len(right_dict), learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cfeb1357",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_212260/3508881337.py:1: The name tf.train.Saver is deprecated. Please use tf.compat.v1.train.Saver instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_212260/3508881337.py:1: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'lstm/model.ckpt'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'lstm/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4cd7109c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "51b21eaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, train_Y = [], []\n",
    "for i in range(len(left)):\n",
    "    train_X.append([left_dict[c] for c in left[i]] + [1])\n",
    "    train_Y.append([right_dict[c] for c in right[i]] + [1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bf736137",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_x, _ = pad_sentence_batch(train_X[: 5], 0)\n",
    "batch_y, _ = pad_sentence_batch(train_Y[: 5], 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e44c2c00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 14)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(batch_y).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "65f1ab08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 11)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(batch_x).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "67f53b4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 417 ms, sys: 103 ms, total: 520 ms\n",
      "Wall time: 250 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[3.8444717, None]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "sess.run([model.cost, model.optimizer], feed_dict = {model.X: batch_x, model.Y: batch_y})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ff00eb04",
   "metadata": {},
   "outputs": [],
   "source": [
    "greedy, beam = sess.run([model.greedy, model.beam], feed_dict = {model.X: train_X[:1]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6ae9aaba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[7, 1]], dtype=int32), array([[1, 1, 1, 1, 1]], dtype=int32))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "greedy, beam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b006b329",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 32)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epoch, batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "ec0d3145",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 19/19 [00:09<00:00,  2.05it/s, accuracy=1, cost=0.00721]    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 0.009723, training avg acc 0.998853\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 19/19 [00:09<00:00,  1.95it/s, accuracy=1, cost=0.0048]     "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, training avg loss 0.006597, training avg acc 0.999433\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "for e in range(2):\n",
    "    train_X, train_Y = shuffle(train_X, train_Y)\n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc = [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x, seq_x = pad_sentence_batch(train_X[i : index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(train_Y[i : index], PAD)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "6ad3a562",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'lstm-bahdanau/model.ckpt'"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'lstm-bahdanau/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "15f222d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'comel'\n",
    "batch = [left_dict[c] for c in string] + [1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "b8aa7b02",
   "metadata": {},
   "outputs": [],
   "source": [
    "greedy, beam = sess.run([model.greedy, model.beam], feed_dict = {model.X: [batch]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "18a45fc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[34, 15, 38, 28,  7, 15, 39,  7, 27, 10, 28,  1]], dtype=int32),\n",
       " array([[34, 15, 38, 28,  7, 15]], dtype=int32))"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "greedy, beam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "64fdbd7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tS\\x8dm.S«.lAm'"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join([rev_right_dict[i] for i in greedy[0] if i > 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "3f836ac6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_212260/3818740975.py:4: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Variable',\n",
       " 'Variable_1',\n",
       " 'rnn/multi_rnn_cell/cell_0/lstm_cell/kernel',\n",
       " 'rnn/multi_rnn_cell/cell_0/lstm_cell/bias',\n",
       " 'rnn/multi_rnn_cell/cell_1/lstm_cell/kernel',\n",
       " 'rnn/multi_rnn_cell/cell_1/lstm_cell/bias',\n",
       " 'decode/memory_layer/kernel',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_0/lstm_cell/kernel',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_0/lstm_cell/bias',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_1/lstm_cell/kernel',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_1/lstm_cell/bias',\n",
       " 'decode/decoder/attention_wrapper/bahdanau_attention/query_layer/kernel',\n",
       " 'decode/decoder/attention_wrapper/bahdanau_attention/attention_v',\n",
       " 'decode/decoder/attention_wrapper/attention_layer/kernel',\n",
       " 'decode/decoder/dense/kernel',\n",
       " 'decode/decoder/dense/bias',\n",
       " 'decode_1/greedy',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/beam_width',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/range/start',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/range/delta',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/range',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/mul/y',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/mul',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/ExpandDims/dim',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/ExpandDims',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/add',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape/shape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape_1/shape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape_1',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/GatherV2/axis',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/GatherV2',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Shape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice/stack',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice/stack_1',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice/stack_2',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/output',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_word_ids/y',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_word_ids',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_parent_ids',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_finished',\n",
       " 'decode_2/beam']"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'greedy' in n.name\n",
    "        or 'beam' in n.name\n",
    "        or 'alphas' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'OptimizeLoss' not in n.name\n",
    "        and 'Global_Step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "41334eee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            \"directory: %s\" % model_dir)\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "    \n",
    "    absolute_model_dir = \"/\".join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + \"/frozen_model.pb\"\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph=tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices)\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(\",\")\n",
    "        ) \n",
    "        with tf.gfile.GFile(output_graph, \"wb\") as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print(\"%d ops in the final graph.\" % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "9fda0e24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_212260/1070649395.py:3: The name tf.gfile.Exists is deprecated. Please use tf.io.gfile.exists instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_212260/1070649395.py:14: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_212260/1070649395.py:15: The name tf.train.import_meta_graph is deprecated. Please use tf.compat.v1.train.import_meta_graph instead.\n",
      "\n",
      "INFO:tensorflow:Restoring parameters from lstm-bahdanau/model.ckpt\n",
      "INFO:tensorflow:Froze 16 variables.\n",
      "INFO:tensorflow:Converted 16 variables to const ops.\n",
      "WARNING:tensorflow:From /tmp/ipykernel_212260/1070649395.py:22: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n",
      "1649 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph(\"lstm-bahdanau\", strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "8d4adfc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, \"rb\") as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "9f937cb1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_212260/3576390908.py:3: The name tf.GraphDef is deprecated. Please use tf.compat.v1.GraphDef instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "g=load_graph('lstm-bahdanau/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "997bd77d",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "i_greedy = g.get_tensor_by_name('import/decode_1/greedy:0')\n",
    "i_beam = g.get_tensor_by_name('import/decode_2/beam:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "22a981d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/tf-nvidia/lib/python3.8/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "test_sess = tf.InteractiveSession(graph=g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "967a8820",
   "metadata": {},
   "outputs": [],
   "source": [
    "greedy, beam = test_sess.run([i_greedy, i_beam], feed_dict = {x: [batch]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "7a3d54c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tS\\x8dm.S«.lAm'"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join([rev_right_dict[i] for i in greedy[0] if i > 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "08e7ca99",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "from tensorflow.contrib.seq2seq.python.ops import beam_search_ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "48c7848d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-30 21:36:49.682102: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying add_default_attributes\n",
      "2022-05-30 21:36:49.687661: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying remove_nodes\n",
      "2022-05-30 21:36:49.693479: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_1/greedy\n",
      "2022-05-30 21:36:49.694247: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_2/beam\n",
      "2022-05-30 21:36:49.701659: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_1/greedy\n",
      "2022-05-30 21:36:49.702217: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_2/beam\n",
      "2022-05-30 21:36:49.708453: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_1/greedy\n",
      "2022-05-30 21:36:49.709027: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_2/beam\n",
      "2022-05-30 21:36:49.728517: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying fold_batch_norms\n",
      "2022-05-30 21:36:49.737639: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying fold_old_batch_norms\n",
      "2022-05-30 21:36:49.761065: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying quantize_weights\n",
      "2022-05-30 21:36:49.798518: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying strip_unused_nodes\n",
      "2022-05-30 21:36:49.804359: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying sort_by_execution_order\n"
     ]
    }
   ],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']\n",
    "\n",
    "pb = 'lstm-bahdanau/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       ['Placeholder'],\n",
    "                                       ['decode_1/greedy', 'decode_2/beam'], transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "261c7832",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lstm-bahdanau/\r\n",
      "lstm-bahdanau/checkpoint\r\n",
      "lstm-bahdanau/frozen_model.pb.quantized\r\n",
      "lstm-bahdanau/model.ckpt.index\r\n",
      "lstm-bahdanau/model.ckpt.data-00000-of-00001\r\n",
      "lstm-bahdanau/model.ckpt.meta\r\n",
      "lstm-bahdanau/frozen_model.pb\r\n"
     ]
    }
   ],
   "source": [
    "!tar -cvf phoneme-lstm-bahdanau.tar lstm-bahdanau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "28ace921",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/tf-nvidia/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from malaya_boilerplate.huggingface import upload_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "8c6f9ccd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/tf-nvidia/lib/python3.8/site-packages/huggingface_hub/hf_api.py:79: FutureWarning: `name` and `organization` input arguments are deprecated and will be removed in v0.8. Pass `repo_id` instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "files_mapping = {'lstm-bahdanau/frozen_model.pb': 'model.pb'}\n",
    "upload_dict(model = 'phoneme-lstm-bahdanau', files_mapping = files_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "1cbd614c",
   "metadata": {},
   "outputs": [],
   "source": [
    "files_mapping = {'lstm-bahdanau/frozen_model.pb.quantized': 'model.pb'}\n",
    "upload_dict(model = 'phoneme-lstm-bahdanau-quantized', files_mapping = files_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "f7bcbd6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "files_mapping = {'phoneme-lstm-bahdanau.tar': 'phoneme-lstm-bahdanau.tar'}\n",
    "upload_dict(model = 'pretrained-phoneme', files_mapping = files_mapping)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf1",
   "language": "python",
   "name": "tf1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
